import requests
from bs4 import BeautifulSoup as bs
import threading
import queue

# da le tou


class get_history(threading.Thread):
    def __init__(self, task_q, result_q):
        super().__init__()
        self.task_queue = task_q
        self.result_queue = result_q

    def run(self):
        while True:
            if not self.task_queue.empty():
                page = self.task_queue.get()
                one_result = self.crawl(page)
                self.result_queue.put(one_result)
                self.task_queue.task_done()
                print('##第{}页爬取完毕~~~~~'.format(page))
            else:
                break

    def crawl(self, page):
        url = 'http://www.lottery.gov.cn/historykj/history_{}.jspx?_ltype=dlt'.format(
            page)
        headers = {
            'user-agent': 'Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 75.0.3770.142Safari / 537.36',
            'Upgrade-Insecure-Requests': '1',
            'Host': 'www.lottery.gov.cn'
        }
        r = requests.get(url, headers=headers)
        text = bs(r.text, 'lxml')
        result = text.find_all('div', class_='result')
        result = result[0].find_all('tbody')
        result = result[0].find_all('td')
        result_list = []
        for item in result:
            result_list.append(item.get_text())
        one_page = []
        one_page.append([result_list[19]] + result_list[0:8])
        for i in range(1, 20):
            open_data = result_list[19 + (20 * i)]
            number_list = result_list[20 +
                                      (20 * (i - 1)):21 + (20 * (i - 1)) + 7]
#            print([open_data] + number_list)
            one_page.append([open_data] + number_list)
        return one_page


if __name__ == '__main__':
    task_queue = queue.Queue()
    result_queue = queue.Queue()
    for i in range(1, 94):
        task_queue.put(i)
    crawl = get_history(task_queue, result_queue)
    crawl.setDaemon(True)
    crawl.start()

    task_queue.join()
    with open('history.txt', 'a') as f:

        while not result_queue.empty():
            for one in result_queue.get():
                one_line = ''
                for item in one:
                    # one_line += item
                    one_line += item
                    one_line += ' '
                f.write(one_line+'\n')
